# -*- coding: utf-8 -*-


import sys
import re
import requests
import json

from urlparse import urljoin
from scrapy import Request, FormRequest
from scrapy.spider import Spider
from bs4 import BeautifulSoup

from realEstateCommittee.items import RealestatecommitteeItem

reload(sys)
sys.setdefaultencoding('utf8')


class RealEstateCommittee_JXM_NC(Spider):
    name = "RealEstateCommittee_JXM_NC"

    start_urls = ["http://www.ncfdj.gov.cn/projectQuery/search.jspx"]

    def make_requests_from_url(self, url):
        headers = dict()
        meta = dict()
        headers[
            "User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36"

        return Request(url, callback=self.parse, headers=headers, meta=meta, dont_filter=True)

    PROJECT_DETAIL_URL = "project_detail_url"
    BUILDINGS_URL = "buildings_url"
    HOUSE_URL = "house_url"
    HOUSE_DETAIL_URL = "house_detail_url"

    REGIID = "regiId"

    PROJECT = "project"
    PROJECT_URL = "project_url"
    BELONG_CITY = "belong_city"  #: "所在区县",
    PROJECT_ADDR = "project_addr"  #: "项目地址",
    PRESALE_PERMIT = "presale_permit"  #:"预售证号码",
    RESIDENCE_SET = "residence_set"  #:"本期住宅总套数",
    RESIDENCE_AREA = "residence_area"  #:"本期住宅总面积",
    PROPERTY_TYPE = "property_type"  #:"物业类型/房屋类型",
    BUILDING_TYPE = "building_type"  #:"建筑类型",
    OPEN_TIME = "open_time"  #: "开盘时间",
    COMPLETE_TIME = "complete_time"  #: "竣工时间",
    DEVELOPER = "developer"  #:"开发商名称",
    BUILDING_NO = "building_no"  #:"楼号/幢号",
    UNIT_NO = "unit_no"  #:"单元号",
    FLOOR = "floor"  #:"楼层",
    HOUSE_NO = "house_no"  #:"房号",
    SALE_STATUS = "sale_status"  #:"销售状态"

    def parse(self, response):
        meta = response.meta
        bs_obj = BeautifulSoup(response.text, "html5lib")

        """
        PRESALE_PERMIT = "presale_permit" #:"预售证号码",           预售许可证号   
        PROJECT = "project" #                                                   
        PROJECT_ADDR = "project_addr" #: "项目地址",                项目地址       
        DEVELOPER = "developer" #:"开发商名称",                      开发企业   
        RESIDENCE_SET = "residence_set" #:"本期住宅总套数",           总套数      
        PROJECT_URL = "project_url" #     
        BUILDINGS_URL = "buildings_url"                                       
        """
        for row in bs_obj.find("table", class_="tab-list ").find_all("tr")[1:]:
            meta[self.PRESALE_PERMIT] = row.find_all("td")[0].get_text()
            meta[self.PROJECT] = row.find_all("td")[1].get_text()
            meta[self.PROJECT_ADDR] = row.find_all("td")[2].get_text()
            meta[self.DEVELOPER] = row.find_all("td")[3].get_text()
            meta[self.RESIDENCE_SET] = row.find_all("td")[4].get_text()
            meta[self.PROJECT_URL] = urljoin(response.url, row.find_all("a")[0].attrs["href"])
            meta[self.BUILDINGS_URL] = urljoin(response.url, row.find_all("a")[1].attrs["href"])

            self.logger.info("预售证号码:%s,项目名称:%s,项目地址:%s,开发商名称:%s,本期住宅总套数:%s,项目链接:%s,楼盘链接:%s" %
                (meta[self.PRESALE_PERMIT], meta[self.PROJECT], meta[self.PROJECT_ADDR], meta[self.DEVELOPER],
                 meta[self.RESIDENCE_SET], meta[self.PROJECT_URL], meta[self.BUILDINGS_URL]))

            yield Request(meta[self.PROJECT_URL], callback=self.parse_project, meta=meta, dont_filter=True)

        next_button = bs_obj.find("a", attrs={"title":u"下一页", "disabled":None})
        next_button = None
        if next_button:
            next_url = "http://www.ncfdj.gov.cn/projectQuery/search.jspx?page=%s" % re.findall("'(.*?)'", next_button.attrs["onclick"])[0]
            self.logger.info("存在下一页，链接为：%s" % next_url)
            yield Request(next_url, meta=meta, dont_filter=True)
        else:
            self.logger.info("不存在下一页")

    def parse_project(self, response):
        meta = response.meta
        bs_obj = BeautifulSoup(response.text, "html5lib")

        """
        BELONG_CITY = "belong_city" #: "所在区县",                  行政区       
        OPEN_TIME = "open_time" #: "开盘时间",                      开工日期        
        COMPLETE_TIME = "complete_time" #: "竣工时间",              竣工时间        
        PROPERTY_TYPE = "property_type" #:"物业类型/房屋类型",         房屋类型     
        BUILDING_TYPE = "building_type" #:"建筑类型",               建筑类型        
        RESIDENCE_AREA = "residence_area" #:"本期住宅总面积",         建筑面积     
        """

        table_row = bs_obj.find("table", class_="tab-list table-view").find_all("tr")
        meta[self.BELONG_CITY] = table_row[3].find_all("td")[1].get_text()
        meta[self.OPEN_TIME] = table_row[4].find_all("td")[1].get_text()
        meta[self.COMPLETE_TIME] = table_row[4].find_all("td")[3].get_text()
        meta[self.PROPERTY_TYPE] = table_row[5].find_all("td")[1].get_text()
        meta[self.BUILDING_TYPE] = table_row[5].find_all("td")[3].get_text()
        meta[self.RESIDENCE_AREA] = table_row[6].find_all("td")[5].get_text()

        self.logger.info("所在区县：%s, 本期住宅总面积：%s, 物业类型/房屋类型：%s, 建筑类型：%s, 开盘时间：%s, 竣工时间：%s" % (meta[self.BELONG_CITY], meta[self.OPEN_TIME], meta[self.COMPLETE_TIME], meta[self.PROPERTY_TYPE], meta[self.BUILDING_TYPE], meta[self.RESIDENCE_AREA]))

        yield Request(meta[self.BUILDINGS_URL], callback=self.parse_buildings, meta=meta, dont_filter=True)

    def parse_buildings(self, response):
        """
        BUILDING_NO = "building_no" #:"楼号/幢号",                  楼号/幢号
        UNIT_NO = "unit_no" #:"单元号",                            单元号
        FLOOR = "floor" #:"楼层",                                     楼层
        HOUSE_NO = "house_no" #:"房号",                           房号
        SALE_STATUS = "sale_status" #:"销售状态"                    销售状态
        :param response:
        :return:
        """

        meta = response.meta

        meta[self.REGIID] = re.findall("=(.*?)&", meta[self.BUILDINGS_URL])
        data_1 = {
            "regiId": meta[self.REGIID]
        }
        yield FormRequest("http://www.ncfdj.gov.cn/projectQuery/getBuildings.jspx", formdata=data_1, callback=self.parse_house, meta=meta, dont_filter=True)

    def parse_house(self, response):
        meta = response.meta

        building_json = json.loads(response.text)
        for building in building_json["data"]:
            meta[self.BUILDING_NO] = building["buildingNumber"]
            for unit in building["unitList"]:
                meta[self.UNIT_NO] = unit["unitId"] if unit["unitId"] else ""
                data_2 = {
                    "buildUuid": building["uuid"],
                    "unitId": meta[self.UNIT_NO],
                    "id": meta[self.REGIID]
                }

                yield FormRequest("http://www.ncfdj.gov.cn/projectQuery/getHouResource.jspx", formdata=data_2,
                                  callback=self.parse_houseinfo, meta=meta, dont_filter=True)

    def parse_houseinfo(self, response):
        meta = response.meta

        unit_json = json.loads(response.text)
        for house in unit_json["data"]:
            meta[self.FLOOR] = house["floor"]
            meta[self.HOUSE_NO] = house["number"]
            meta[self.SALE_STATUS] = house["status"]
            self.logger.info("楼号:%s, 单元号:%s, 楼层:%s, 房号:%s, 销售状态:%s" % (meta[self.BUILDING_NO], meta[self.UNIT_NO], meta[self.FLOOR], meta[self.HOUSE_NO], meta[self.SALE_STATUS]))
            yield self.meta_to_item(meta)

    def meta_to_item(self, meta):
        item = RealestatecommitteeItem()
        for field in RealestatecommitteeItem.fields:
            if field not in meta.keys():
                item[field] = ""
            else:
                item[field] = meta[field]

        return item